{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "from framework import NamedDatasetSpecifications, EvaluationDatasetSampling, FlowTransformer, FlowTransformerParameters\n",
        "from implementations.classification_heads import *\n",
        "from implementations.input_encodings import *\n",
        "from implementations.pre_processings import StandardPreProcessing\n",
        "from implementations.transformers.basic_transformers import BasicTransformer\n",
        "from implementations.transformers.named_transformers import *\n"
      ],
      "metadata": {
        "id": "g3B-oLAzBnyO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encodings = [\n",
        "    NoInputEncoder(),\n",
        "    RecordLevelEmbed(64),\n",
        "    CategoricalFeatureEmbed(EmbedLayerType.Dense, 16),\n",
        "    CategoricalFeatureEmbed(EmbedLayerType.Lookup, 16),\n",
        "    CategoricalFeatureEmbed(EmbedLayerType.Projection, 16),\n",
        "    RecordLevelEmbed(64, project=True)\n",
        "]\n",
        "\n",
        "classification_heads = [\n",
        "    LastTokenClassificationHead(),\n",
        "    FlattenClassificationHead(),\n",
        "    GlobalAveragePoolingClassificationHead(),\n",
        "    CLSTokenClassificationHead(),\n",
        "    FeaturewiseEmbedding(project=False),\n",
        "    FeaturewiseEmbedding(project=True),\n",
        "]\n",
        "\n",
        "transformers = [\n",
        "    BasicTransformer(2, 128, n_heads=2),\n",
        "    BasicTransformer(2, 128, n_heads=2, is_decoder=True),\n",
        "    GPTSmallTransformer(),\n",
        "    BERTSmallTransformer()\n",
        "]"
      ],
      "metadata": {
        "id": "_48j9v64BwxC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! mkdir dataset"
      ],
      "metadata": {
        "id": "9l_zeRW_nDW9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! mv /content/drive/MyDrive/NF-UNSW-NB15-v2.csv dataset/."
      ],
      "metadata": {
        "id": "V1vVv2ninGaB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "flow_file_path = r\"/content/dataset\"\n",
        "\n",
        "datasets = [\n",
        "    (\"UNSW_NB15\", os.path.join(flow_file_path, \"NF-UNSW-NB15-v2.csv\"), NamedDatasetSpecifications.unified_flow_format, 0.025, EvaluationDatasetSampling.LastRows)\n",
        "]\n"
      ],
      "metadata": {
        "id": "3imgDX7VCdkU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_processing = StandardPreProcessing(n_categorical_levels=32)\n",
        "\n",
        "# Define the transformer\n",
        "ft = FlowTransformer(pre_processing=pre_processing,\n",
        "                     input_encoding=encodings[0],\n",
        "                     sequential_model=transformers[0],\n",
        "                     classification_head=classification_heads[0],\n",
        "                     params=FlowTransformerParameters(window_size=8, mlp_layer_sizes=[128], mlp_dropout=0.1))\n",
        "\n",
        "# Load the specific dataset\n",
        "cache_folder = '/content/cache_folder'\n",
        "dataset_name, dataset_path, dataset_specification, eval_percent, eval_method = datasets[0]\n",
        "ft.load_dataset(dataset_name, dataset_path, dataset_specification, cache_folder,evaluation_dataset_sampling=eval_method, evaluation_percent=eval_percent)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "GrDBlDxgDPV1",
        "outputId": "a08f9c3a-411a-4c36-f40f-6fd6cc6c9687"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cache file path: /content/cache_folder/UNSW_NB15_0_QdLmZHuh8yOmlGcKBEkf7hepImY0_VzQ981ONg0PHPxtLtik6rZN0dGw0.feather\n",
            "Attempting to read dataset from path /content/dataset/NF-UNSW-NB15-v2.csv...\n",
            "Set y to = Attack\n",
            "Converting numerical columns to floats, and removing out of range values...\n",
            "Applying pre-processing to numerical values\n",
            "[Numerical 1 / 28] Processing numerical column LONGEST_FLOW_PKT...\n",
            "[Numerical 2 / 28] Processing numerical column DURATION_IN...\n",
            "[Numerical 3 / 28] Processing numerical column IN_PKTS...\n",
            "[Numerical 4 / 28] Processing numerical column NUM_PKTS_128_TO_256_BYTES...\n",
            "[Numerical 5 / 28] Processing numerical column MAX_TTL...\n",
            "[Numerical 6 / 28] Processing numerical column OUT_PKTS...\n",
            "[Numerical 7 / 28] Processing numerical column MIN_IP_PKT_LEN...\n",
            "[Numerical 8 / 28] Processing numerical column NUM_PKTS_256_TO_512_BYTES...\n",
            "[Numerical 9 / 28] Processing numerical column IN_BYTES...\n",
            "[Numerical 10 / 28] Processing numerical column FLOW_DURATION_MILLISECONDS...\n",
            "[Numerical 11 / 28] Processing numerical column RETRANSMITTED_OUT_BYTES...\n",
            "[Numerical 12 / 28] Processing numerical column TCP_WIN_MAX_OUT...\n",
            "[Numerical 13 / 28] Processing numerical column SHORTEST_FLOW_PKT...\n",
            "[Numerical 14 / 28] Processing numerical column OUT_BYTES...\n",
            "[Numerical 15 / 28] Processing numerical column SRC_TO_DST_SECOND_BYTES...\n",
            "[Numerical 16 / 28] Processing numerical column DURATION_OUT...\n",
            "[Numerical 17 / 28] Processing numerical column RETRANSMITTED_IN_BYTES...\n",
            "[Numerical 18 / 28] Processing numerical column NUM_PKTS_UP_TO_128_BYTES...\n",
            "[Numerical 19 / 28] Processing numerical column NUM_PKTS_512_TO_1024_BYTES...\n",
            "[Numerical 20 / 28] Processing numerical column MIN_TTL...\n",
            "[Numerical 21 / 28] Processing numerical column TCP_WIN_MAX_IN...\n",
            "[Numerical 22 / 28] Processing numerical column RETRANSMITTED_IN_PKTS...\n",
            "[Numerical 23 / 28] Processing numerical column RETRANSMITTED_OUT_PKTS...\n",
            "[Numerical 24 / 28] Processing numerical column DST_TO_SRC_AVG_THROUGHPUT...\n",
            "[Numerical 25 / 28] Processing numerical column MAX_IP_PKT_LEN...\n",
            "[Numerical 26 / 28] Processing numerical column SRC_TO_DST_AVG_THROUGHPUT...\n",
            "[Numerical 27 / 28] Processing numerical column DST_TO_SRC_SECOND_BYTES...\n",
            "[Numerical 28 / 28] Processing numerical column NUM_PKTS_1024_TO_1514_BYTES...\n",
            "Applying pre-processing to categorical values\n",
            "[Categorical 1 / 9] Processing categorical column CLIENT_TCP_FLAGS...\n",
            "Encoding the 16 levels for CLIENT_TCP_FLAGS\n",
            "[Categorical 2 / 9] Processing categorical column L4_SRC_PORT...\n",
            "Encoding the 32 levels for L4_SRC_PORT\n",
            "[Categorical 3 / 9] Processing categorical column TCP_FLAGS...\n",
            "Encoding the 17 levels for TCP_FLAGS\n",
            "[Categorical 4 / 9] Processing categorical column ICMP_IPV4_TYPE...\n",
            "Encoding the 32 levels for ICMP_IPV4_TYPE\n",
            "[Categorical 5 / 9] Processing categorical column ICMP_TYPE...\n",
            "Encoding the 32 levels for ICMP_TYPE\n",
            "[Categorical 6 / 9] Processing categorical column PROTOCOL...\n",
            "Encoding the 32 levels for PROTOCOL\n",
            "[Categorical 7 / 9] Processing categorical column SERVER_TCP_FLAGS...\n",
            "Encoding the 15 levels for SERVER_TCP_FLAGS\n",
            "[Categorical 8 / 9] Processing categorical column L4_DST_PORT...\n",
            "Encoding the 32 levels for L4_DST_PORT\n",
            "[Categorical 9 / 9] Processing categorical column L7_PROTO...\n",
            "Encoding the 32 levels for L7_PROTO\n",
            "Generating pre-processed dataframe...\n",
            "Input data frame had shape (2390275,45), output data frame has shape (2390275,270) after pre-processing...\n",
            "Writing to cache file path: /content/cache_folder/UNSW_NB15_0_QdLmZHuh8yOmlGcKBEkf7hepImY0_VzQ981ONg0PHPxtLtik6rZN0dGw0.feather...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         LONGEST_FLOW_PKT  DURATION_IN   IN_PKTS  NUM_PKTS_128_TO_256_BYTES  \\\n",
              "0                0.565324          0.0  0.000000                   0.000000   \n",
              "1                0.565324          0.0  0.160324                   0.000000   \n",
              "2                0.575948          0.0  0.218877                   0.000000   \n",
              "3                0.598516          0.0  0.255508                   0.000000   \n",
              "4                0.598516          0.0  0.293311                   0.000000   \n",
              "...                   ...          ...       ...                        ...   \n",
              "2390270          0.915812          0.0  0.247534                   0.192724   \n",
              "2390271          0.565324          0.0  0.069048                   0.000000   \n",
              "2390272          0.999258          0.0  0.229372                   0.000000   \n",
              "2390273          0.984754          0.0  0.420327                   0.000000   \n",
              "2390274          0.984754          0.0  0.351279                   0.192724   \n",
              "\n",
              "          MAX_TTL  OUT_PKTS  MIN_IP_PKT_LEN  NUM_PKTS_256_TO_512_BYTES  \\\n",
              "0        0.630549  0.148859        0.608608                   0.000000   \n",
              "1        0.630549  0.223288        0.608608                   0.000000   \n",
              "2        0.630549  0.266827        0.608608                   0.000000   \n",
              "3        0.630549  0.297718        0.608608                   0.000000   \n",
              "4        0.630549  0.331913        0.608608                   0.000000   \n",
              "...           ...       ...             ...                        ...   \n",
              "2390270  0.630549  0.290788        0.608608                   0.000000   \n",
              "2390271  0.625000  0.117968        0.659772                   0.000000   \n",
              "2390272  0.630549  0.257484        0.608608                   0.132745   \n",
              "2390273  0.630549  0.457722        0.608608                   0.000000   \n",
              "2390274  0.630549  0.375452        0.608608                   0.000000   \n",
              "\n",
              "         IN_BYTES  FLOW_DURATION_MILLISECONDS  ...  L7_PROTO_23  L7_PROTO_24  \\\n",
              "0        0.127562                         0.0  ...            0            0   \n",
              "1        0.323054                         0.0  ...            0            0   \n",
              "2        0.358547                         0.0  ...            0            0   \n",
              "3        0.380413                         0.0  ...            0            0   \n",
              "4        0.402810                         0.0  ...            0            0   \n",
              "...           ...                         ...  ...          ...          ...   \n",
              "2390270  0.404639                         0.0  ...            0            0   \n",
              "2390271  0.289329                         0.0  ...            0            0   \n",
              "2390272  0.400688                         0.0  ...            0            0   \n",
              "2390273  0.481723                         0.0  ...            0            0   \n",
              "2390274  0.448013                         0.0  ...            0            0   \n",
              "\n",
              "         L7_PROTO_25  L7_PROTO_26  L7_PROTO_27  L7_PROTO_28  L7_PROTO_29  \\\n",
              "0                  0            0            0            0            0   \n",
              "1                  0            0            0            0            0   \n",
              "2                  0            0            0            0            0   \n",
              "3                  0            0            0            0            0   \n",
              "4                  0            0            0            0            0   \n",
              "...              ...          ...          ...          ...          ...   \n",
              "2390270            0            0            0            0            0   \n",
              "2390271            0            0            0            0            0   \n",
              "2390272            0            0            0            0            0   \n",
              "2390273            0            0            0            0            0   \n",
              "2390274            0            0            0            0            0   \n",
              "\n",
              "         L7_PROTO_30  L7_PROTO_31  L7_PROTO_32  \n",
              "0                  0            0            0  \n",
              "1                  0            0            0  \n",
              "2                  0            0            0  \n",
              "3                  0            0            0  \n",
              "4                  0            0            0  \n",
              "...              ...          ...          ...  \n",
              "2390270            0            0            0  \n",
              "2390271            0            0            0  \n",
              "2390272            0            0            0  \n",
              "2390273            0            0            0  \n",
              "2390274            0            0            0  \n",
              "\n",
              "[2390275 rows x 268 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-886df5fd-a9ec-462e-bf63-5388f1d06241\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>LONGEST_FLOW_PKT</th>\n",
              "      <th>DURATION_IN</th>\n",
              "      <th>IN_PKTS</th>\n",
              "      <th>NUM_PKTS_128_TO_256_BYTES</th>\n",
              "      <th>MAX_TTL</th>\n",
              "      <th>OUT_PKTS</th>\n",
              "      <th>MIN_IP_PKT_LEN</th>\n",
              "      <th>NUM_PKTS_256_TO_512_BYTES</th>\n",
              "      <th>IN_BYTES</th>\n",
              "      <th>FLOW_DURATION_MILLISECONDS</th>\n",
              "      <th>...</th>\n",
              "      <th>L7_PROTO_23</th>\n",
              "      <th>L7_PROTO_24</th>\n",
              "      <th>L7_PROTO_25</th>\n",
              "      <th>L7_PROTO_26</th>\n",
              "      <th>L7_PROTO_27</th>\n",
              "      <th>L7_PROTO_28</th>\n",
              "      <th>L7_PROTO_29</th>\n",
              "      <th>L7_PROTO_30</th>\n",
              "      <th>L7_PROTO_31</th>\n",
              "      <th>L7_PROTO_32</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.565324</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.148859</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.127562</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.565324</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.160324</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.223288</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.323054</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.575948</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.218877</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.266827</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.358547</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.598516</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.255508</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.297718</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.380413</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.598516</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.293311</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.331913</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.402810</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2390270</th>\n",
              "      <td>0.915812</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.247534</td>\n",
              "      <td>0.192724</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.290788</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.404639</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2390271</th>\n",
              "      <td>0.565324</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.069048</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.117968</td>\n",
              "      <td>0.659772</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.289329</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2390272</th>\n",
              "      <td>0.999258</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.229372</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.257484</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.132745</td>\n",
              "      <td>0.400688</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2390273</th>\n",
              "      <td>0.984754</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.420327</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.457722</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.481723</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2390274</th>\n",
              "      <td>0.984754</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.351279</td>\n",
              "      <td>0.192724</td>\n",
              "      <td>0.630549</td>\n",
              "      <td>0.375452</td>\n",
              "      <td>0.608608</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.448013</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2390275 rows × 268 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-886df5fd-a9ec-462e-bf63-5388f1d06241')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-886df5fd-a9ec-462e-bf63-5388f1d06241 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-886df5fd-a9ec-462e-bf63-5388f1d06241');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4f231f02-0c2b-4bde-bf5d-b035bad6b6d7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4f231f02-0c2b-4bde-bf5d-b035bad6b6d7')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4f231f02-0c2b-4bde-bf5d-b035bad6b6d7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "m = ft.build_model()\n",
        "m.summary()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CJZH9EX7lGBj",
        "outputId": "6ee264d3-bdc1-43d5-dc46-0711d75156ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/content/implementations/transformers/basic/encoder_block.py:72: UserWarning: Typically inner_dimension should be greater than or equal to the input_dimension!\n",
            "  warnings.warn(f\"Typically inner_dimension should be greater than or equal to the input_dimension!\")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_LONGEST_FLOW_PKT (InputL  [(None, 8, 1)]      0           []                               \n",
            " ayer)                                                                                            \n",
            "                                                                                                  \n",
            " input_DURATION_IN (InputLayer)  [(None, 8, 1)]      0           []                               \n",
            "                                                                                                  \n",
            " input_IN_PKTS (InputLayer)     [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_NUM_PKTS_128_TO_256_BYTE  [(None, 8, 1)]      0           []                               \n",
            " S (InputLayer)                                                                                   \n",
            "                                                                                                  \n",
            " input_MAX_TTL (InputLayer)     [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_OUT_PKTS (InputLayer)    [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_MIN_IP_PKT_LEN (InputLay  [(None, 8, 1)]      0           []                               \n",
            " er)                                                                                              \n",
            "                                                                                                  \n",
            " input_NUM_PKTS_256_TO_512_BYTE  [(None, 8, 1)]      0           []                               \n",
            " S (InputLayer)                                                                                   \n",
            "                                                                                                  \n",
            " input_IN_BYTES (InputLayer)    [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_FLOW_DURATION_MILLISECON  [(None, 8, 1)]      0           []                               \n",
            " DS (InputLayer)                                                                                  \n",
            "                                                                                                  \n",
            " input_RETRANSMITTED_OUT_BYTES   [(None, 8, 1)]      0           []                               \n",
            " (InputLayer)                                                                                     \n",
            "                                                                                                  \n",
            " input_TCP_WIN_MAX_OUT (InputLa  [(None, 8, 1)]      0           []                               \n",
            " yer)                                                                                             \n",
            "                                                                                                  \n",
            " input_SHORTEST_FLOW_PKT (Input  [(None, 8, 1)]      0           []                               \n",
            " Layer)                                                                                           \n",
            "                                                                                                  \n",
            " input_OUT_BYTES (InputLayer)   [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_SRC_TO_DST_SECOND_BYTES   [(None, 8, 1)]      0           []                               \n",
            " (InputLayer)                                                                                     \n",
            "                                                                                                  \n",
            " input_DURATION_OUT (InputLayer  [(None, 8, 1)]      0           []                               \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " input_RETRANSMITTED_IN_BYTES (  [(None, 8, 1)]      0           []                               \n",
            " InputLayer)                                                                                      \n",
            "                                                                                                  \n",
            " input_NUM_PKTS_UP_TO_128_BYTES  [(None, 8, 1)]      0           []                               \n",
            "  (InputLayer)                                                                                    \n",
            "                                                                                                  \n",
            " input_NUM_PKTS_512_TO_1024_BYT  [(None, 8, 1)]      0           []                               \n",
            " ES (InputLayer)                                                                                  \n",
            "                                                                                                  \n",
            " input_MIN_TTL (InputLayer)     [(None, 8, 1)]       0           []                               \n",
            "                                                                                                  \n",
            " input_TCP_WIN_MAX_IN (InputLay  [(None, 8, 1)]      0           []                               \n",
            " er)                                                                                              \n",
            "                                                                                                  \n",
            " input_RETRANSMITTED_IN_PKTS (I  [(None, 8, 1)]      0           []                               \n",
            " nputLayer)                                                                                       \n",
            "                                                                                                  \n",
            " input_RETRANSMITTED_OUT_PKTS (  [(None, 8, 1)]      0           []                               \n",
            " InputLayer)                                                                                      \n",
            "                                                                                                  \n",
            " input_DST_TO_SRC_AVG_THROUGHPU  [(None, 8, 1)]      0           []                               \n",
            " T (InputLayer)                                                                                   \n",
            "                                                                                                  \n",
            " input_MAX_IP_PKT_LEN (InputLay  [(None, 8, 1)]      0           []                               \n",
            " er)                                                                                              \n",
            "                                                                                                  \n",
            " input_SRC_TO_DST_AVG_THROUGHPU  [(None, 8, 1)]      0           []                               \n",
            " T (InputLayer)                                                                                   \n",
            "                                                                                                  \n",
            " input_DST_TO_SRC_SECOND_BYTES   [(None, 8, 1)]      0           []                               \n",
            " (InputLayer)                                                                                     \n",
            "                                                                                                  \n",
            " input_NUM_PKTS_1024_TO_1514_BY  [(None, 8, 1)]      0           []                               \n",
            " TES (InputLayer)                                                                                 \n",
            "                                                                                                  \n",
            " input_CLIENT_TCP_FLAGS (InputL  [(None, 8, 16)]     0           []                               \n",
            " ayer)                                                                                            \n",
            "                                                                                                  \n",
            " input_L4_SRC_PORT (InputLayer)  [(None, 8, 32)]     0           []                               \n",
            "                                                                                                  \n",
            " input_TCP_FLAGS (InputLayer)   [(None, 8, 17)]      0           []                               \n",
            "                                                                                                  \n",
            " input_ICMP_IPV4_TYPE (InputLay  [(None, 8, 32)]     0           []                               \n",
            " er)                                                                                              \n",
            "                                                                                                  \n",
            " input_ICMP_TYPE (InputLayer)   [(None, 8, 32)]      0           []                               \n",
            "                                                                                                  \n",
            " input_PROTOCOL (InputLayer)    [(None, 8, 32)]      0           []                               \n",
            "                                                                                                  \n",
            " input_SERVER_TCP_FLAGS (InputL  [(None, 8, 15)]     0           []                               \n",
            " ayer)                                                                                            \n",
            "                                                                                                  \n",
            " input_L4_DST_PORT (InputLayer)  [(None, 8, 32)]     0           []                               \n",
            "                                                                                                  \n",
            " input_L7_PROTO (InputLayer)    [(None, 8, 32)]      0           []                               \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 8, 268)       0           ['input_LONGEST_FLOW_PKT[0][0]', \n",
            "                                                                  'input_DURATION_IN[0][0]',      \n",
            "                                                                  'input_IN_PKTS[0][0]',          \n",
            "                                                                  'input_NUM_PKTS_128_TO_256_BYTES\n",
            "                                                                 [0][0]',                         \n",
            "                                                                  'input_MAX_TTL[0][0]',          \n",
            "                                                                  'input_OUT_PKTS[0][0]',         \n",
            "                                                                  'input_MIN_IP_PKT_LEN[0][0]',   \n",
            "                                                                  'input_NUM_PKTS_256_TO_512_BYTES\n",
            "                                                                 [0][0]',                         \n",
            "                                                                  'input_IN_BYTES[0][0]',         \n",
            "                                                                  'input_FLOW_DURATION_MILLISECOND\n",
            "                                                                 S[0][0]',                        \n",
            "                                                                  'input_RETRANSMITTED_OUT_BYTES[0\n",
            "                                                                 ][0]',                           \n",
            "                                                                  'input_TCP_WIN_MAX_OUT[0][0]',  \n",
            "                                                                  'input_SHORTEST_FLOW_PKT[0][0]',\n",
            "                                                                  'input_OUT_BYTES[0][0]',        \n",
            "                                                                  'input_SRC_TO_DST_SECOND_BYTES[0\n",
            "                                                                 ][0]',                           \n",
            "                                                                  'input_DURATION_OUT[0][0]',     \n",
            "                                                                  'input_RETRANSMITTED_IN_BYTES[0]\n",
            "                                                                 [0]',                            \n",
            "                                                                  'input_NUM_PKTS_UP_TO_128_BYTES[\n",
            "                                                                 0][0]',                          \n",
            "                                                                  'input_NUM_PKTS_512_TO_1024_BYTE\n",
            "                                                                 S[0][0]',                        \n",
            "                                                                  'input_MIN_TTL[0][0]',          \n",
            "                                                                  'input_TCP_WIN_MAX_IN[0][0]',   \n",
            "                                                                  'input_RETRANSMITTED_IN_PKTS[0][\n",
            "                                                                 0]',                             \n",
            "                                                                  'input_RETRANSMITTED_OUT_PKTS[0]\n",
            "                                                                 [0]',                            \n",
            "                                                                  'input_DST_TO_SRC_AVG_THROUGHPUT\n",
            "                                                                 [0][0]',                         \n",
            "                                                                  'input_MAX_IP_PKT_LEN[0][0]',   \n",
            "                                                                  'input_SRC_TO_DST_AVG_THROUGHPUT\n",
            "                                                                 [0][0]',                         \n",
            "                                                                  'input_DST_TO_SRC_SECOND_BYTES[0\n",
            "                                                                 ][0]',                           \n",
            "                                                                  'input_NUM_PKTS_1024_TO_1514_BYT\n",
            "                                                                 ES[0][0]',                       \n",
            "                                                                  'input_CLIENT_TCP_FLAGS[0][0]', \n",
            "                                                                  'input_L4_SRC_PORT[0][0]',      \n",
            "                                                                  'input_TCP_FLAGS[0][0]',        \n",
            "                                                                  'input_ICMP_IPV4_TYPE[0][0]',   \n",
            "                                                                  'input_ICMP_TYPE[0][0]',        \n",
            "                                                                  'input_PROTOCOL[0][0]',         \n",
            "                                                                  'input_SERVER_TCP_FLAGS[0][0]', \n",
            "                                                                  'input_L4_DST_PORT[0][0]',      \n",
            "                                                                  'input_L7_PROTO[0][0]']         \n",
            "                                                                                                  \n",
            " block_0_transformer_encoder (T  (None, 8, 268)      345544      ['concatenate[0][0]']            \n",
            " ransformerEncoderBlock)                                                                          \n",
            "                                                                                                  \n",
            " block_1_transformer_encoder (T  (None, 8, 268)      345544      ['block_0_transformer_encoder[0][\n",
            " ransformerEncoderBlock)                                         0]']                             \n",
            "                                                                                                  \n",
            " slice_last (Lambda)            (None, 268)          0           ['block_1_transformer_encoder[0][\n",
            "                                                                 0]']                             \n",
            "                                                                                                  \n",
            " classification_mlp_0_128 (Dens  (None, 128)         34432       ['slice_last[0][0]']             \n",
            " e)                                                                                               \n",
            "                                                                                                  \n",
            " dropout (Dropout)              (None, 128)          0           ['classification_mlp_0_128[0][0]'\n",
            "                                                                 ]                                \n",
            "                                                                                                  \n",
            " binary_classification_out (Den  (None, 1)           129         ['dropout[0][0]']                \n",
            " se)                                                                                              \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 725,649\n",
            "Trainable params: 725,649\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "m.compile(optimizer=\"adam\", loss='binary_crossentropy', metrics=['binary_accuracy'], jit_compile=True)\n",
        "\n",
        "# Get the evaluation results\n",
        "eval_results: pd.DataFrame\n",
        "(train_results, eval_results, final_epoch) = ft.evaluate(m, batch_size=128, epochs=5, steps_per_epoch=64, early_stopping_patience=5)\n",
        "\n",
        "print(eval_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bnAQhGeInWl8",
        "outputId": "75c3e3b9-cbf2-4d58-b0e2-718db1984f5a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Building eval dataset...\n",
            "Splitting dataset to featurewise...\n",
            "Evaluation dataset is built!\n",
            "Positive samples in eval set: 2816\n",
            "Negative samples in eval set: 56940\n",
            "Epoch = 0 / 5 (early stop in 5), step = 0, loss = 0.70432, results = [0.7043224573135376, 0.4921875] -- elapsed (train): 0.00s\n",
            "Epoch = 0 / 5 (early stop in 5), step = 31, loss = 0.12169, results = [0.12169043719768524, 0.9609375] -- elapsed (train): 1.02s\n",
            "Epoch = 0 / 5 (early stop in 5), step = 63, loss = 0.01447, results = [0.014470361173152924, 1.0] -- elapsed (train): 2.02s\n",
            "Epoch = 1 / 5 (early stop in 5), step = 28, loss = 0.08485, results = [0.08485106378793716, 0.9765625] -- elapsed (train): 2.96s\n",
            "Epoch = 1 / 5 (early stop in 5), step = 50, loss = 0.08263, results = [0.08262961357831955, 0.984375] -- elapsed (train): 3.88s\n",
            "Epoch = 2 / 5 (early stop in 5), step = 18, loss = 0.00547, results = [0.005471301265060902, 1.0] -- elapsed (train): 4.87s\n",
            "Epoch = 2 / 5 (early stop in 5), step = 49, loss = 0.00368, results = [0.00368143105879426, 1.0] -- elapsed (train): 5.84s\n",
            "Epoch = 3 / 5 (early stop in 5), step = 17, loss = 0.00197, results = [0.0019733035005629063, 1.0] -- elapsed (train): 6.83s\n",
            "Epoch = 3 / 5 (early stop in 5), step = 35, loss = 0.04993, results = [0.049926236271858215, 0.9921875] -- elapsed (train): 7.63s\n",
            "Epoch = 3 / 5 (early stop in 5), step = 63, loss = 0.00721, results = [0.007205802947282791, 1.0] -- elapsed (train): 8.59s\n",
            "Epoch = 4 / 5 (early stop in 5), step = 30, loss = 0.04294, results = [0.042935848236083984, 0.984375] -- elapsed (train): 9.54s\n",
            "Epoch = 4 / 5 (early stop in 5), step = 61, loss = 0.00428, results = [0.004281854256987572, 1.0] -- elapsed (train): 10.49s\n",
            "1868/1868 [==============================] - 10s 4ms/step\n",
            "Epoch 4 yielded predictions: (59756,), overall balanced accuracy: 99.28%, TP = 2,810 / 2,816, TN = 56,243 / 56,940\n",
            "   epoch     P      N  pred_P  pred_N    TP   FP     TN  FN   bal_acc  \\\n",
            "0      4  2816  56940    3507   56249  2810  697  56243   6  0.992814   \n",
            "\n",
            "         f1  \n",
            "0  0.888819  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "qLtSy_P_nfSx"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}